# -*- coding: utf-8 -*-

import json
import redis,hashlib

class RedisPipeline(object):
    def process_item(self, item, spider):
        try:
            r = redis.StrictRedis(host='', port=6379,password='',db=1)
            md5 = hashlib.md5()
            md5.update(item['link_url'].encode(encoding='utf-8'))
            url_md5 = md5.hexdigest()
            if not r.hexists('url_md5', key=url_md5):
                r.hset('url_md5', key=url_md5, value=1)
                r.rpush('link_url', item['link_url'])
            else:
                print('md5值已存在')

            return item
        except:
            print('没有连接到redis')


